clear
set mem 1000m

capture log close
log using "$dataloc/log/bankscope_data", text replace
set more off

/* --------------------------------------------------------------- *
   Author: Ed Al-Hussainy
   Description:  Process raw BankScope data  
 * --------------------------------------------------------------- */
 
/* --------------------------------------------------------------- *
    Read-in raw data
 * --------------------------------------------------------------- */
 
* USE: readmerger A B C D
* Program reads in all files with extension A in directory B, 
* merges them using variables C and saves a single output file D

capture program drop readmerger
program readmerger
    version 9.1  
    args filext dir mergevars outds 

    quietly {

        local list : dir "`dir'" files "`filext'" , nofail
        local listn : word count `list'
        global bsvars ""

        cd "`dir'"  

        forvalues i = 1/`listn' {

            local fil : word `i' of `list'
            local ser = subinstr("`fil'",".csv","",.)
            global bsvars = trim("$bsvars data`ser'")

            noisily di "Currently processing `fil' (file `i' of `listn')"

            insheet using "`fil'", names clear
            capture drop bankname
            drop if missing(indexnumber)
            
            local idvars " col1 v1 bankname specialisationgeneral bvdepidnumber countryname countrycode conscode status indexnumber isinnumber tickersymbol sedolnumber valornumber ciknumber"   
            ds  
            local allvars "`r(varlist)'"
            local datavars : list allvars - idvars 
            
            local y = 1990
            foreach v in `datavars' { 
               rename `v' data`ser'`y'
               local y = `y' + 1
            }   
                       
            bysort `mergevars': gen _t = _n
            drop if _t > 1
            drop _t

            * isid `mergevars'

            * clean up and rename variables                
            foreach j of var _all {                   
                capture replace `j' = "" if `j' == "n.a."
                capture destring `j', replace
            }              
            if `i' == 1 {
                save `outds', replace
            }
            else if `listn' > 1 {
                merge `mergevars' using `outds', sort
                noisily tab _merge
                keep if _merge == 3
                drop _merge* 
                save `outds', replace
            }
        } 
    }    
end

readmerger *.csv "$dataloc/Data/BankScope/$dataver" "indexnumber" BankScopeRaw
capture drop v1 col1

* use "$dataloc/Data/BankScope/$dataver/BankScopeRaw", clear

/* --------------------------------------------------------------- *
    Apply filters
 * --------------------------------------------------------------- */

* General Specialization
keep if inlist(specialisationgeneral,"Commercial Banks")

* Statement Type : Unconsolidated preferred to Consolidated
keep if inlist(conscode,"U1","U2","C1")

/* --------------------------------------------------------------- *
    Transpose dataset
 * --------------------------------------------------------------- */

global bsvars = subinstr("$bsvars","dataheader","",.)
di "BankScope series: $bsvars"

keep countrycode indexnumber data*

reshape long $bsvars, i(countrycode indexnumber) j(year) str

destring year, replace

isid indexnumber year

sort countrycode

/* --------------------------------------------------------------- *
    Translate country codes
 * --------------------------------------------------------------- */

merge countrycode using "$dataloc/Data/BankScope/BSCountryCodes", uniqusing
tab _merge
keep if _merge == 3
drop _merge

/* --------------------------------------------------------------- *
    save results
 * --------------------------------------------------------------- */

keep indexnumber cncode year data*
isid indexnumber year

compress
save "$dataloc/Data/BankScope/$dataver/BankScope", replace

capture erase "$dataloc/Data/BankScope/$dataver/BankScopeRaw.dta"

/* end of code */
clear
log close

